import requests
import os

url = 'https://www.baidu.com/robots.txt'
r = requests.get(url);
print(type(r))
# print(r.text[:100])
if r.status_code == 200:
   open('robots.txt','w').write(r.text)



